/*
 * Copyright (c) 2010 CCX(China) Co.,Ltd. All Rights Reserved.
 *
 * This software is the confidential and proprietary information of
 * CCX(China) Co.,Ltd. ("Confidential Information").
 * It may not be copied or reproduced in any manner without the express 
 * written permission of CCX(China) Co.,Ltd.
 * 
 *	Created on 2010-12-24 上午11:47:59
 */
package cn.com.ccxe.core.crawl.listing;

import java.util.HashSet;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import cn.com.ccxe.core.entity.ArticleLink;
import cn.com.ccxe.core.entity.ListPageLink;
import cn.com.ccxe.core.entity.WebData;

/**
 * 内容提取器。<br>
 * @author hetao
 */
public class ContentExtractor {

	Document document ;
	final Logger logger = LoggerFactory.getLogger(ContentExtractor.class);
	
	public ContentExtractor(WebData data) {
		document = Jsoup.parse(data.getHtml(), data.getLink().getPath());
	}
	
	/**
	 * 根据规则，抽取html代码中的列表页的链接。
	 * @param selectExpr	抽取表达式
	 * @return			
	 * 					符合抽取规则的HTML代码
	 */
	public Set<ListPageLink> extractorListPage(String selectExpr) {
		Set<ListPageLink> set = new HashSet<ListPageLink>();
		Elements elements = document.select(selectExpr);
		return findListPageLink(set, elements);
		
	}
	
	private Set<ListPageLink> findListPageLink(Set<ListPageLink> set, Elements elements) {
		for (Element e : elements) {
			if("a".equalsIgnoreCase(e.tagName())) {
				ListPageLink link = new ListPageLink();
				String href = e.absUrl("href");
				if(!StringUtils.isEmpty(href)) {
					link.setPath(href);
					set.add(link);
				}
				continue;
			}
			Elements aTags = e.select("a");
			//递归查找 A 标签
			findListPageLink(set, aTags);		
		}
		return set;
	}
	
	/**
	 * 根据规则抽取文章链接
	 * @param selectExpr
	 * @return
	 */
	public Set<ArticleLink> extractorArticleList(String selectExpr) {
		Set<ArticleLink> set = new HashSet<ArticleLink>();
		Elements elements = document.select(selectExpr);
		return findArticleLink(set, elements);
	}
	
	private Set<ArticleLink> findArticleLink(Set<ArticleLink> set, Elements elements) {
		for (Element e : elements) {
			if("a".equalsIgnoreCase(e.tagName())) {
				ArticleLink link = new ArticleLink();
				String href = e.absUrl("href");
				if(href.indexOf("?") > 0 || href.indexOf("blog") > 0) {
					//FIXME URL标准化
					continue;
				}
				if(!StringUtils.isEmpty(href)) {
					link.setPath(href);
					link.setTitle(e.text());
					set.add(link);
				}
				continue;
			}
			Elements aTags = e.select("a");
			//递归查找 A 标签
			findArticleLink(set, aTags);		
		}
		return set;
	}
	
	
	
	
}
